//-L ../components_lib  -Wl,--gc-sections
#include <iostream>
#include <stdexcept>
#include <fstream>
#include <cctype>
#include <algorithm>
#include <vector>
#include <cassert>
#include <string>
#include <deque>

// boost
#include <boost/foreach.hpp>

// components
#include <print_error.h>
#include <digest.h>
#include <progress_indicator.h>
#include <md4.h>
#include <crc.h>
#define DEBUG_LEVEL 9
#include <debug_func.h>
#include <seq_pairs.h>


#include "get_arg.h"

using std::vector;
using std::ofstream;
using std::deque;
using std::string;
using std::cerr;
using std::cout;
using std::sort;
using std::ifstream;

// get substring which consists of first two ids
void parse_string_pair(const string& orig, string& id1, string& id2)
{
	string::const_iterator i, end;
	i = orig.begin();
	end = orig.end();
	i = std::find(i,  end,  '\t');
	if (i == end)
		throw std::runtime_error("Invalid data (without tab-separated identifiers) ");
	id1.assign(orig.begin(), i);
	if (id1.empty())
		throw std::runtime_error("Missing identifier");
	id2.assign(++i, end);
}




struct t_digest_pos
{
	t_digest			digest;
	std::ios::streampos file_pos;
	string				identifier;
	t_digest_pos(std::ios::streampos file_pos_, const string& identifier_) : 
				file_pos(file_pos_),
				identifier(identifier_){	}
	bool operator <(const t_digest_pos& o) const;
};
inline bool t_digest_pos::operator <(const t_digest_pos& o) const
{
	if (digest == o.digest)
		return file_pos < o.file_pos;
	return digest < o.digest;
}


unsigned check_lines(ifstream& f_input, t_digest_pos dp1, t_digest_pos dp2,
					 ofstream& f_identical, ofstream& f_collisions)
{
	f_input.clear();
	f_input.seekg(dp1.file_pos, std::ios::beg);
	string line1;
	if (!getline(f_input, line1))
		throw std::runtime_error("Could not read line from " + ultoa(dp1.file_pos));
	string id1, data1;
	parse_string_pair(line1, id1, data1);

	f_input.clear();
	f_input.seekg(dp2.file_pos, std::ios::beg);
	string line2;
	if (!getline(f_input, line2))
		throw std::runtime_error("Could not read line from " + ultoa(dp2.file_pos));
	string id2, data2;
	parse_string_pair(line2, id2, data2);


	if (data1 == data2)
	{
		f_identical << "1\t" << id1 << "\t" << dp1.file_pos << "\t" << dp1.digest.single_field_str() << line1 << "\n";
		f_identical << "2\t" << id2 << "\t" << dp2.file_pos << "\t" << dp2.digest.single_field_str() << line2 << "\n";
		return 0;
	}
	else
	{
		f_collisions << "1\t" << id1 << "\t" << dp1.file_pos << "\t" << dp1.digest.single_field_str() << line1 << "\n";
		f_collisions << "2\t" << id2 << "\t" << dp2.file_pos << "\t" << dp2.digest.single_field_str() << line2 << "\n";
		return 1;
	}
}



//8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888

//	main

//8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
int main (int argc, char *argv[])
{
	debug_func_cerr(5);
	// get command line arguments

	// if no arguments print usage and exit
	t_program_args args;
	try
	{
		if (!process_command_line_options(argc, argv, args))
			return 1;

		deque<t_digest_pos> arr_digest;
		{
			// progress dots
			unsigned dots_size = 5000;
			t_progress_indicator dots(args.VERBOSE(), dots_size);
			dots.use_timer();
	
	
			// hash everything
			std::ios::streampos curr_pos (0);
			string line;
			args.VERBOSE() << "Reading file [" + args.input_file + "]\n";
			while (getline(args.f_input, line))
			{
				string id1, data;
				parse_string_pair(line, id1, data);
				
				arr_digest.push_back(t_digest_pos(curr_pos, id1));
				md4_hash(data,  arr_digest.back().digest);
				cout << id1 << "\t" << arr_digest.back().digest.single_field_str() << "\n";
	
				// save position for next line
				curr_pos = args.f_input.tellg();
	
				++dots;
				// check every ten dots 
				if (dots.value() % (dots_size * 10) == 0)
				{
					unsigned dots_needed = unsigned(dots.value() * 1.0 *
													args.input_file_size / curr_pos);
					dots_needed += dots_size;
					dots.set_limit(dots_needed);
				}
				
			}
		}

		args.VERBOSE() << "Read " << arr_digest.size() << " lines\n";
		args.VERBOSE() << "Sorting...\n";
		sort(arr_digest.begin(),  arr_digest.end());

		ofstream f_identical(args.identical_file.c_str());
		ofstream f_collision(args.collision_file.c_str());
		if (!f_identical)
			throw std::runtime_error("Couldn't open " + args.identical_file + "\n");
		if (!f_collision)
			throw std::runtime_error("Couldn't open " + args.collision_file + "\n");
		
		unsigned cnt_collisions = 0;
		unsigned cnt_identical = 0;
		{
			args.VERBOSE() << "Counting collisions...\n";
			// progress dots
			unsigned dots_size = arr_digest.size() / 200;
			t_progress_indicator dots(args.VERBOSE(), dots_size);
			dots.use_timer();


			for (unsigned i = 0; i < arr_digest.size() - 1; ++i)
			{
				++dots;
				if (arr_digest[i].digest == arr_digest[i + 1].digest)
				{
					++cnt_identical;
					cnt_collisions += check_lines(args.f_input, arr_digest[i], 
												  arr_digest[i + 1],
												  f_identical,
												  f_collision);
				}
			}
		}
		args.VERBOSE() << "There were " << cnt_identical << " identical lines\n";
		args.VERBOSE() << "There were " << cnt_collisions << " collisions\n";

	}
	catch(std::exception& e)
	{
		std_print_error(args.CERR(), e.what());
		return 1;
	}


	return(0);
}


